# -*- coding: utf-8 -*-
import scrapy
from ..items import XiaoshuoItem


class XiaoshuoSpider(scrapy.Spider):
    name = 'xiaoshuo'
    allowed_domains = ['www.qb5.tw']
    start_urls = ['http://www.qb5.tw']

    # 一级解析：获取主题
    def parse(self, response):
        topic_list = response.xpath('/html/body/div[2]/div[1]/ul/li[position()>1 and position()<9]')
        for i in topic_list:
            xiaoshuo_topic = i.xpath('./a/@title').get()
            link = i.xpath('./a/@href').get()
            yield scrapy.Request(url=link,meta={'xiaoshuo_topic': xiaoshuo_topic},callback=self.parse_page)

    # 二级解析：遍历页码数
    def parse_page(self, response):
        xiaoshuo_topic = response.meta['xiaoshuo_topic']
        #<a href="https://www.qb5.tw/fenlei/1_376/" class="last">376</a>
        page_count = response.xpath('//*[@id="pagelink"]/a[@class="last"]/text()').get()
        url = response.url
        page_count = 1
        for i in (1, page_count + 1, 1):
            link = url[:-2]+str(i)+"/"
            yield scrapy.Request(
                url=link,
                meta={'xiaoshuo_topic': xiaoshuo_topic},
                callback=self.parse_two
            )

    # 二级解析：获取主题下小说
    def parse_two(self, response):
        xiaoshuo_topic = response.meta['xiaoshuo_topic']
        novle_lists = response.xpath('//div[@class="zp"]')
        for i in novle_lists:
            xiaoshuo_name = i.xpath('./a/@title').get()
            link = i.xpath('./a/@href').get()
            yield scrapy.Request(
                url=link,
                meta={'xiaoshuo_topic': xiaoshuo_topic, 'xiaoshuo_name': xiaoshuo_name,"requestUrl":link},
                callback=self.parse_three
            )

    # 三级解析：获取小说章节
    def parse_three(self, response):
        xiaoshuo_topic = response.meta['xiaoshuo_topic']
        xiaoshuo_name = response.meta['xiaoshuo_name']
        requestUrl = response.meta['requestUrl']
        chapter_list = response.xpath('/html/body/div[@class="zjbox"]/dl/dd')
        for i in chapter_list:
            xiaoshuo_chapter = i.xpath('./a/text()').get()
            link = str(requestUrl) + i.xpath('./a/@href').get()
            yield scrapy.Request(
                url=link,
                meta={'xiaoshuo_topic': xiaoshuo_topic, 'xiaoshuo_name': xiaoshuo_name,'xiaoshuo_chapter': xiaoshuo_chapter},
                callback=self.parse_fuor
            )

    # 四级解析：获取小说内容
    def parse_fuor(self, response):
        xiaoshuo_topic = response.meta['xiaoshuo_topic']
        xiaoshuo_name = response.meta['xiaoshuo_name']
        xiaoshuo_chapter = response.meta['xiaoshuo_chapter']

        item = XiaoshuoItem()
        item['xiaoshuo_topic'] = xiaoshuo_topic
        item['xiaoshuo_chapter'] = xiaoshuo_chapter
        item['xiaoshuo_name'] = xiaoshuo_name
        item['xiaoshuo_content'] = response.xpath('//div[@id="content"]/text()').extract()

        yield item
